from URLX import URL  as Url,URLError as UrlError,InvalidURL as InvalidUrl
from URLX import  url_split
DOMAIN=set(['com','net','org','gov',
		'cn','us','tw','uk',
	])
def url2site(u):
	#return 3-tuple:(site,sub,host)
	t=url_split(u)
	host=t[1]
	tokens=host.split('.')
	size=len(tokens)
	for idx in range(size-1,-1,-1):
		if tokens[idx] not in DOMAIN:
			subS=idx-1
			if subS<0:
				subS=0
			subE=idx+1
			return tokens[idx],'.'.join(tokens[subS:subE]),host
	return None
	
url_to_site=url_site=url2site
def normalize_url(u):
        u=u.strip()
        if u[-1]=='/':
                u=u[:-1]
        return u
def test_1():
	print url_site('http://www.sohu.com')
	print url_site('http://www.sohu.cn')
	print url_site('http://sohu.cn')
	print url_site('http://sohu.com.cn')
	print url_site('http://sohu.net.cn')
	print url_site('http://news.sohu.com.cn')
	print url_site('http://news.sohu.com.cn/index.html')
	print url_site('http://news.sohu.com.cn/pic/3.jpg')
	print url_site('http://qq.news.sohu.com.cn/pic/3.jpg')
def test_2():
	print url_site('www.sohu.com')
	print url_site('htp://www.sohu.com')
	print url_site('htp//www.sohu.com')

if __name__=='__main__':
	test_1()
	#test_2()
